
import urllib
import urllib2
import xml.dom.minidom


try:
    from google.appengine.api.urlfetch import fetch
    from StringIO import StringIO

    def urlopen(url, query):
        return StringIO(fetch(url + "?" + query).content)
    
    urllib2.urlopen = urlopen
except ImportError:
    pass



class WikipediaAPI(object):
    
    WIKIPEDIA_API_URL = 'http://en.wikipedia.org/w/api.php'
    
    class Article(object):
        def __init__(self, title):
            self.title = title
    
    def search(self, srsearch, srwhat='text'):
        query = urllib.urlencode({'action': 'query',
                                  'list': 'search',
                                  'srsearch': srsearch,
                                  'srwhat': srwhat,
                                  'format': 'xml'})
        xmlret = urllib2.urlopen(self.WIKIPEDIA_API_URL,
                                 query).read()
        xmlret_dom = xml.dom.minidom.parseString(xmlret)
        return [x.getAttribute("title") for x in 
                xmlret_dom.getElementsByTagName("p")]

    def get_article(self, titles):
        if not isinstance(titles, list):
            titles = [titles]
        query = dict(action='query',
                     prop='revisions',
                     titles='|'.join(titles).encode("utf-8"),
                     rvprop='content',
                     format='xml')
        xmlret = urllib2.urlopen(self.WIKIPEDIA_API_URL,
                                 urllib.urlencode(query)).read()
        xmlret_dom = xml.dom.minidom.parseString(xmlret)
        pages_by_title = \
            dict((page.getAttribute("title"), page)
                 for page in xmlret_dom.getElementsByTagName("page"))
        def mkarticle(title, text):
            a = self.Article(title)
            a.content = text
            return a
        return [mkarticle(title, n.firstChild.wholeText)
                for title in titles
                    for n in pages_by_title[title].getElementsByTagName("rev")]
        
        
class WikipediaMining(object):
    
    def __init__(self):
        self.wikipedia = WikipediaAPI()

    def harvest_math(self, search_string):
        search = self.wikipedia.search(search_string, "text")
        for article in self.wikipedia.get_article(search):
            yield((article.title,
                   self.get_math(article.content)))

    def get_math(self, content):
        import re
        math = re.compile("<math>(.*?)</math>", re.MULTILINE | re.DOTALL)
        return [mo.group(1) for mo in math.finditer(content)]
            
        
        
        
                                
if __name__ == '__main__':
    mine = WikipediaMining()
    for article, formulas in mine.harvest_math("quadratic equation"):
        print article
